{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3e8c3908-a502-4371-bd0c-52810a58d689",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from abc import ABC, abstractmethod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e8b4b698-23a8-4b0c-a0ea-7eb21fab5fe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BernoulliBandit():\n",
    "    def __init__(self, k):\n",
    "        self.probs = np.random.uniform(size = k)\n",
    "        self.max_idx = np.argmax(self.probs)\n",
    "        self.max_n = self.probs[self.max_idx]\n",
    "        self.k = k\n",
    "    def step(self, k):\n",
    "            if np.random.rand() < self.probs[k]:\n",
    "                return 1\n",
    "            else:\n",
    "                return 0\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "bd2b319b-fc01-4eac-8b14-48df738e8e69",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Solver(ABC):\n",
    "    def __init__(self, bandit):\n",
    "        self.bandit = bandit\n",
    "        self.counts = np.zeros(self.bandit.k) \n",
    "        self.actions = []\n",
    "        self.regret = 0\n",
    "        self.regrets = []\n",
    "        \n",
    "    @abstractmethod\n",
    "    def run_step(self):\n",
    "        pass\n",
    "\n",
    "    def update_regret(self, k):\n",
    "        self.regret += self.bandit.max_n -  self.bandit.probs[k]\n",
    "        self.regrets.append(self.regret)\n",
    "\n",
    "    def run(self, T):\n",
    "        for _ in range(T):\n",
    "            k = self.run_step()\n",
    "            self.counts[k] += 1\n",
    "            self.update_regret(k)\n",
    "            self.actions.append(k)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "18fbbf03-11c7-47c4-9c8c-eb0bcb901d35",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EpsilonGreedy(Solver):\n",
    "    def __init__(self, bandit ,epsilon=0.01, init_prob = 1.0):\n",
    "        self.bandit = bandit\n",
    "        self.epsilon = epsilon\n",
    "        self.estimates = np.array([init_prob] * self.bandit.k)\n",
    "\n",
    "    def run_step(self):\n",
    "        # 选择期望最大的拉杆\n",
    "        if np.random.rand() > epsilon:\n",
    "            k = np.argmax(estimates)\n",
    "        else:\n",
    "            k = np.random.randint(0, self.bandit.k)\n",
    "        r = self.bandit.step(k)  # 得到本次动作的奖励\n",
    "        self.estimates[k] += 1. / (self.counts[k] + 1) * (r - self.estimates[k])\n",
    "\n",
    "        return k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2859181e-c771-45f3-a4d5-15599da2ecf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1)\n",
    "K = 10\n",
    "bandit_10_arm = BernoulliBandit(K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "67c057d1-d085-45f5-bf50-201815e9c7fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7203244934421581"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bandit_10_arm.max_n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
